Import Libraries and Packages

import numpy as np
import cv2
import glob
import joblib
from copy import copy
import matplotlib.pyplot as plt
#import sklearn
from skimage import exposure
from skimage.io import imread
from scipy.ndimage.measurements import label

Get Spatial, Colour and Hog Features

# Change color-space of image before getting feature vector
def change_color_space(image, colour_space):
    # YUV: color encoding system Y is "luma" or brightness, U and V are Blue minus luma and Red minus luma
    if colour_space == 'YUV':
        img = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
    # YCrCb: used in digital systems similar to YUV, are used interchangeably with subtle difference in the way they are encoded
    elif colour_space == 'YCrCb':
        img = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
    else:
        img = image
    return img


# Get spatial features "raw pixels" through resizing image, convert each channel to column vector then stack the 3 channels
def get_spatial_features(image, size=(32, 32)):
    channel_1 = cv2.resize(image[:, :, 0], size).ravel()            # For size (32, 32), it produces (1024, ) vector
    channel_2 = cv2.resize(image[:, :, 1], size).ravel()
    channel_3 = cv2.resize(image[:, :, 2], size).ravel()
    features = np.hstack((channel_1, channel_2, channel_3))         # For size (32, 32), it produces (3072, ) vector
    return features


# Get color features using histogram of each channel and concatenating them
def get_colour_features(image, bins=32):                     # bins is the number of discrete values in the histogram
    channel_1 = np.histogram(image[:, :, 0], bins=bins)
    channel_2 = np.histogram(image[:, :, 1], bins=bins)
    channel_3 = np.histogram(image[:, :, 2], bins=bins)
    features = np.concatenate((channel_1[0], channel_2[0], channel_3[0]))
    return features


# Get HOG features and turn it to column vector
def get_hog_features(image):
    # windowSize, blockSize,blockStride,cellSize,nbins
    # windowSize: the size of window sliding over the image
    # blockSize: a block contain multiple cells, we normalize gradient over blocks (size is in pixels)
    # cellSize: a histogram is created for each cell then it is normalized over the block (size is in pixels)
    # blockStride: as cell size
    # nbins: the number of discrete orientations
    features = cv2.HOGDescriptor((64, 64), (16, 16), (8, 8), (8, 8), 9).compute(image)
    return features

Get Prediction Features

def get_features(image, colour_space='YUV', size=(64, 64), type='All'):                # feature type is All or HOG
    if size != (64, 64):                                        # resize image to get features
        image = cv2.resize(image, (64, 64))
    image = change_color_space(image, colour_space)             # change colour_space of the image
    if type == 'All':
        image_features = []
        image_features.append(get_spatial_features(image))
        image_features.append(get_colour_features(image))
        image_features.append(get_hog_features(image))
        features = np.concatenate(image_features)
    else:
        features = get_hog_features(image)
    return features

Generate Heatmap

buffer_weights = [0.1, 0.2, 0.3, 0.4]

def generate_heatmap(image, windows_list):
    heat = np.zeros_like(image[:, :, 0]).astype(np.float)  # use blank dark image
    # Add heat to each box in box list
    heat = add_heat(heat, windows_list)
    # Apply threshold to help remove false positives
    heat = apply_threshold(heat, 1)
    # Visualize the heatmap when displaying
    heatmap = np.clip(heat, 0, 255)

    return heatmap


def add_heat(heatmap, bbox_list):
    # Iterate through list of bboxes
    for box in bbox_list:
        # Add += 1 for all pixels inside each bbox
        # Assuming each "box" takes the form ((x1, y1), (x2, y2))
        heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1      # box[0][1]:y1     box[1][1]:y2

    # Return updated heatmap
    return heatmap  # Iterate through list of bboxes


def apply_threshold(heatmap, threshold):
    # Zero out pixels below the threshold
    heatmap[heatmap <= threshold] = 0       # threshold is determined as 1 or 2
    # Return thresholded map
    return heatmap


# New--------------------------
def draw_boxes(img, bboxes, color=(0, 0, 255), thick=6):
    out_img = np.copy(img)
    for b in bboxes:
        cv2.rectangle(out_img, *b, color, thick)  # drawing rectangle that is found in the pointer
    return out_img

# ------------------------------


def draw_bboxes(img, heatmap_buffer, heatmap_pre, N_buffer, prev_bboxes, new_frame, mask):
    if new_frame:
        heatmap_buffer.append(heatmap_pre)

        if len(heatmap_buffer) > N_buffer:  # remove the first component if it is more than N_buffer elements
            heatmap_buffer.pop(0)

        # weight the heatmap based on current frame and previous N frames
        idxs = range(N_buffer)
        for b, w, idx in zip(heatmap_buffer, buffer_weights, idxs):
            heatmap_buffer[idx] = b * w

        heatmap = np.sum(np.array(heatmap_buffer), axis=0)
        heatmap = apply_threshold(heatmap, threshold=sum(buffer_weights[0:N_buffer])*2)

        # Find final boxes from heatmap using label function
        labels = label(heatmap)

        bboxes = []
        # locate the bounding box
        for car_number in range(1, labels[1]+1):
            # Find pixels with each car_number label value
            nonzero = (labels[0] == car_number).nonzero()
            # Identify x and y values of those pixels
            nonzeroy = np.array(nonzero[0])
            nonzerox = np.array(nonzero[1])
            # Define a bounding box based on min/max x and y
            bbox_tmp = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
            bboxes.append(bbox_tmp)

        # New ------------------------
        out = np.copy(img)

        # creating a mask having only the blue boxes with thickness =
        mask = draw_boxes(np.zeros_like(img), bboxes=bboxes, thick=-1)

        # returning contours of the bounded boxes by passing the B channel only
        contours, _ = cv2.findContours(mask[:, :, 2].astype('uint8'), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        for countour in contours:
            x, y, width, height = cv2.boundingRect(countour)
            cv2.rectangle(out, (x, y), (x + width, y + height), (0, 0, 255), 4)

        # ------------------------------

    else:
        out = img
        heatmap = 0
        bboxes = []
    # Return the image
    return out, heatmap, bboxes, mask

Find Cars

mlp = joblib.load('models/MLP, YUV, ALL/mlp_model.pkl')  # loading the multilayer perceptron model
X_scaler = joblib.load('models/MLP, YUV, ALL/scaler.pkl')  # loading a transform for images normalization as in training

def slide_window(img, x_start_stop=[None, None], y_start_stop=[None, None],
                    xy_window=(64, 64), xy_overlap=(0.75, 0.75)):
    # setting default values for the part of the image to be rendered to be the whole image
    if x_start_stop[0] is None:
        x_start_stop[0] = 0
    if x_start_stop[1] is None:
        x_start_stop[1] = img.shape[1]
    if y_start_stop[0] is None:
        y_start_stop[0] = 0
    if y_start_stop[1] is None:
        y_start_stop[1] = img.shape[0]

    # determining the axes range
    xspan = x_start_stop[1] - x_start_stop[0]
    yspan = y_start_stop[1] - y_start_stop[0]

    # determining the step of the window to be slided (in pixels)
    nx_pix_per_step = int(xy_window[0]*(1 - xy_overlap[0]))  # moving 0.25 of the window pixels to ensure overlap
    ny_pix_per_step = int(xy_window[1]*(1 - xy_overlap[1]))

    # calculating the number of windows to cover each axes with the step we decided
    nx_windows = int(xspan/nx_pix_per_step)
    ny_windows = int(yspan/ny_pix_per_step)

    # creating window list and passing on it vertically and horizontally
    # creating a window using the vertices of it in shape ((startx, starty), (endx , endy))
    # as if they are the 2 vertices on a diagonal from top to bottom from left to right
    window_list = []
    for ys in range(ny_windows):
        for xs in range(nx_windows):
            startx = xs * nx_pix_per_step + x_start_stop[0]
            endx = (xs+1) * nx_pix_per_step + x_start_stop[0]
            starty = ys * ny_pix_per_step + y_start_stop[0]
            endy = (ys+1) * ny_pix_per_step + y_start_stop[0]
            window_list.append(((startx, starty), (endx, endy)))
    return window_list


def find_cars(image):

    # creating a list for the windows that have a vehicle detected
    on_windows = []

    # Parameters
    # parameters for passing different window sizes
    pxs = 384 #448 #320
    INCREMENT_SIZE_BY = 16
    PXS_LIMIT = 720
    y_start_stop = [400, 660]  # to search for cars in the lower part of the image #[400, 660]
    xy_overlap = (0.8, 0.8)  # overlap between the windows #(0.8, 0.8)

    ACCEPTANCE_THRESHOLD = .99  # to accept the probability of detected cars more than this probability

    while pxs < PXS_LIMIT:
        #700
        windows = slide_window(image, x_start_stop=[50, None], y_start_stop=y_start_stop, xy_window=(pxs, pxs), xy_overlap=xy_overlap)
        for window in windows:
            # resizing the window to fit like the rest of the training images
            resized = cv2.resize((image[window[0][1]: window[1][1], window[0][0]: window[1][0]]), (64, 64))
            features = get_features(resized).reshape(1, -1)  # reshaping it to get one sample/img unlike in the training

            # applying normalization on the extracted features
            x_scaled = X_scaler.transform(features)

            if resized.shape[0] > 0:
                # check if the model predicts there is car in the window greater than the ACCEPTANCE_THRESHOLD
                if mlp.predict_proba(x_scaled.reshape(1, -1))[0][1] > ACCEPTANCE_THRESHOLD:
                    on_windows.append(window)
        pxs += INCREMENT_SIZE_BY

    return on_windows

C:\Users\Owner\anaconda3\lib\site-packages\sklearn\base.py:338: UserWarning: Trying to unpickle estimator LabelBinarizer from version 1.0.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations
  UserWarning,
C:\Users\Owner\anaconda3\lib\site-packages\sklearn\base.py:338: UserWarning: Trying to unpickle estimator MLPClassifier from version 1.0.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations
  UserWarning,
C:\Users\Owner\anaconda3\lib\site-packages\sklearn\base.py:338: UserWarning: Trying to unpickle estimator StandardScaler from version 1.0.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:
https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations
  UserWarning,

Show Car Thumbnails

def draw_background_highlight(image):

    mask = cv2.rectangle(np.copy(image), (0, 0), (image.shape[1], 155), (0, 0, 0), thickness=cv2.FILLED)
    draw_img = cv2.addWeighted(src1=mask, alpha=0.3, src2=image, beta=0.8, gamma=0)

    return draw_img


def draw_thumbnails(img, window_list):

    cv2.putText(img, 'Detected Vehicles', (4, 37), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 0), 2, cv2.LINE_AA)
    for i, bbox in enumerate(window_list):
        thumbnail = img[bbox[0][1]:bbox[1][1], bbox[0][0]:bbox[1][0]]
        vehicle_thumb = cv2.resize(thumbnail, dsize=(100, 80))
        start_x = 300 + (i+1) * 30 + i * 100
        img[60:140, start_x:start_x + 100, :] = vehicle_thumb

Debug mode

def debug_mode(output_img, mask):
    # binary images should be converted to colored maybe using dstack

    # img_cdf, bin_centers = exposure.cumulative_distribution(heatmap_pre)
    # heatmap_pre = np.interp(heatmap_pre, bin_centers, img_cdf)
    # print(heatmap_pre.shape, heatmap_pre.dtype)

    scale_percent = 25

    # calculate the percent of original dimensions
    width = int(output_img.shape[1] * scale_percent / 100)
    height = int(output_img.shape[0] * scale_percent / 100)
    dsize = (width, height)

    # resizing the 2 images with the same size
    # heatmap_pre_output = cv2.resize(heatmap_pre, dsize)
    mask_output = cv2.resize(mask, dsize)

    # putting the images in foreground, background
    # foreground1, foreground2, background = heatmap_pre_output, mask_output, output_img.copy()
    foreground1, background = mask_output, output_img.copy()

    # blending the images
    alpha = 1
    blended_portion1 = cv2.addWeighted(foreground1, alpha,
                                       background[200:200 + height, 10:10 + width, :],
                                       1-alpha,
                                       0,
                                       background, dtype=cv2.CV_64F)
    background[200:200 + height, 10:10 + width, :] = blended_portion1
    # blended_portion2 = cv2.addWeighted(foreground2, alpha,
    #                                    background[220 + height:220 + 2*height, 10:10 + width, :],
    #                                    1 - alpha,
    #                                    0,
    #                                    background, dtype=cv2.CV_64F)
    # background[220 + height:220 + 2*height, 10:10 + width, :] = blended_portion2
    return background

Plotting

def plt_images(orig_image, orig_title, Debug_image, Debug_title, cmap='gray'):
    # Visualize images
    f, (ax1, ax2) = plt.subplots(1, 2, figsize=(25,10))
    ax1.set_title(orig_title, fontsize=30)
    ax1.imshow(orig_image)
    
    ax2.set_title(Debug_title, fontsize=30)
    ax2.imshow(Debug_image)
    
    
    plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Pipeline

bboxes = []
i = 1

# Parameters for junshengfu_heatmap
Heatmap_buffer = []
N_buffer = 3
heatmap_pre = 0
mask = 0


def pipeline2(image, test_images, debug_mode_on):
    global bboxes
    global i
    global heatmap_pre
    global mask
    global clipped

    if i or test_images:  # test_images: to not drop frames in case of testing images

        windows_list = find_cars(np.copy(image))
        heatmap_pre = generate_heatmap(image, windows_list)
        output_image, heatmap_post, bboxes, mask = draw_bboxes(np.copy(image), copy(Heatmap_buffer), heatmap_pre,
                                                    min(len(Heatmap_buffer) + 1, N_buffer), bboxes, i or test_images, mask)
        i = 0
        if len(Heatmap_buffer) >= N_buffer:
            Heatmap_buffer.pop(0)

    else:

        for bbox in bboxes:
            # Draw the box on the image
            cv2.rectangle(image, bbox[0], bbox[1], (0, 0, 255), 4)
        output_image = image
        i = 1

    output_image = draw_background_highlight(output_image)
    draw_thumbnails(output_image, bboxes)
    if debug_mode_on and len(bboxes) > 0:
        # output_image = debug_mode(output_image,heatmap_pre, mask)
        output_image = debug_mode(output_image, mask)
    return output_image


for image in glob.glob('test_images/*.jpg'):
    #print (image)
    start=image.find('\\')
    end=image.find('.')
    #image = imread(image)
    image1 = cv2.imread(image)
    image1=cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
    output_image = pipeline2(image1, 1, 1)
    plt_images(image1,image[start+1:end],output_image,'Debug mode')
    #plt.imshow(output_image)
    #plt.show()